{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to DataFrames\n",
    "**[Bogumił Kamiński](http://bogumilkaminski.pl/about/), November 20, 2020**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using DataFrames"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Possible pitfalls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Know what is copied when creating a `DataFrame`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th></tr><tr><th></th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><p>3 rows × 5 columns</p><tr><th>1</th><td>0.594692</td><td>0.514567</td><td>0.18404</td><td>0.563166</td><td>0.0321274</td></tr><tr><th>2</th><td>0.437888</td><td>0.0622966</td><td>0.223676</td><td>0.788461</td><td>0.128907</td></tr><tr><th>3</th><td>0.52067</td><td>0.191295</td><td>0.923419</td><td>0.704108</td><td>0.0789202</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5\\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64\\\\\n",
       "\t\\hline\n",
       "\t1 & 0.594692 & 0.514567 & 0.18404 & 0.563166 & 0.0321274 \\\\\n",
       "\t2 & 0.437888 & 0.0622966 & 0.223676 & 0.788461 & 0.128907 \\\\\n",
       "\t3 & 0.52067 & 0.191295 & 0.923419 & 0.704108 & 0.0789202 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×5 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1       \u001b[0m\u001b[1m x2        \u001b[0m\u001b[1m x3       \u001b[0m\u001b[1m x4       \u001b[0m\u001b[1m x5        \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Float64  \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64  \u001b[0m\u001b[90m Float64   \u001b[0m\n",
       "─────┼────────────────────────────────────────────────────\n",
       "   1 │ 0.594692  0.514567   0.18404   0.563166  0.0321274\n",
       "   2 │ 0.437888  0.0622966  0.223676  0.788461  0.128907\n",
       "   3 │ 0.52067   0.191295   0.923419  0.704108  0.0789202"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = DataFrame(rand(3, 5), :auto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = copy(x)\n",
    "x === y # not the same object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = DataFrame(x)\n",
    "x === y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "any(x[!, i] === y[!, i] for i in ncol(x)) # the columns are also not the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = DataFrame(x, copycols=false)\n",
    "x === y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all(x[!, i] === y[!, i] for i in ncol(x)) # the columns are the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th></tr><tr><th></th><th>Int64</th><th>Int64</th></tr></thead><tbody><p>3 rows × 2 columns</p><tr><th>1</th><td>1</td><td>1</td></tr><tr><th>2</th><td>2</td><td>2</td></tr><tr><th>3</th><td>3</td><td>3</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 \\\\\n",
       "\t2 & 2 & 2 \\\\\n",
       "\t3 & 3 & 3 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x     \u001b[0m\u001b[1m y     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     1      1\n",
       "   2 │     2      2\n",
       "   3 │     3      3"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = 1:3; y = [1, 2, 3]; df = DataFrame(x=x,y=y) # the same when creating data frames using kwarg syntax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y === df.y # different object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(UnitRange{Int64}, Array{Int64,1})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "typeof(x), typeof(df.x) # range is converted to a vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y === df[:, :y] # slicing rows always creates a copy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "you can avoid copying by using `copycols=false` keyword argument in functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th></tr><tr><th></th><th>Int64</th><th>Int64</th></tr></thead><tbody><p>3 rows × 2 columns</p><tr><th>1</th><td>1</td><td>1</td></tr><tr><th>2</th><td>2</td><td>2</td></tr><tr><th>3</th><td>3</td><td>3</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& x & y\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 \\\\\n",
       "\t2 & 2 & 2 \\\\\n",
       "\t3 & 3 & 3 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x     \u001b[0m\u001b[1m y     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     1      1\n",
       "   2 │     2      2\n",
       "   3 │     3      3"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = DataFrame(x=x,y=y, copycols=false)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y === df.y # now it is the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "select(df, :y)[!, 1] === y # not the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "select(df, :y, copycols=false)[!, 1] === y # the same"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Do not modify the parent of `GroupedDataFrame` or `view`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p><b>GroupedDataFrame with 2 groups based on key: id</b></p><p><i>First Group (3 rows): id = 1</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th>Int64</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>1</td></tr><tr><th>2</th><td>1</td><td>3</td></tr><tr><th>3</th><td>1</td><td>5</td></tr></tbody></table><p>&vellip;</p><p><i>Last Group (3 rows): id = 2</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th>Int64</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td>2</td><td>2</td></tr><tr><th>2</th><td>2</td><td>4</td></tr><tr><th>3</th><td>2</td><td>6</td></tr></tbody></table>"
      ],
      "text/latex": [
       "GroupedDataFrame with 2 groups based on key: id\n",
       "\n",
       "First Group (3 rows): id = 1\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 \\\\\n",
       "\t2 & 1 & 3 \\\\\n",
       "\t3 & 1 & 5 \\\\\n",
       "\\end{tabular}\n",
       "\n",
       "$\\dots$\n",
       "\n",
       "Last Group (3 rows): id = 2\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 2 & 2 \\\\\n",
       "\t2 & 2 & 4 \\\\\n",
       "\t3 & 2 & 6 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "GroupedDataFrame with 2 groups based on key: id\n",
       "First Group (3 rows): id = 1\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     1      1\n",
       "   2 │     1      3\n",
       "   3 │     1      5\n",
       "⋮\n",
       "Last Group (3 rows): id = 2\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     2      2\n",
       "   2 │     2      4\n",
       "   3 │     2      6"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = DataFrame(id=repeat([1,2], outer=3), x=1:6)\n",
    "g = groupby(x, :id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p><b>GroupedDataFrame with 2 groups based on key: id</b></p><p><i>First Group (3 rows): id = 2</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th>Int64</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td>2</td><td>1</td></tr><tr><th>2</th><td>2</td><td>3</td></tr><tr><th>3</th><td>1</td><td>5</td></tr></tbody></table><p>&vellip;</p><p><i>Last Group (3 rows): id = 2</i></p><table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th>Int64</th><th>Int64</th></tr></thead><tbody><tr><th>1</th><td>2</td><td>2</td></tr><tr><th>2</th><td>2</td><td>4</td></tr><tr><th>3</th><td>2</td><td>6</td></tr></tbody></table>"
      ],
      "text/latex": [
       "GroupedDataFrame with 2 groups based on key: id\n",
       "\n",
       "First Group (3 rows): id = 2\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 2 & 1 \\\\\n",
       "\t2 & 2 & 3 \\\\\n",
       "\t3 & 1 & 5 \\\\\n",
       "\\end{tabular}\n",
       "\n",
       "$\\dots$\n",
       "\n",
       "Last Group (3 rows): id = 2\n",
       "\n",
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 2 & 2 \\\\\n",
       "\t2 & 2 & 4 \\\\\n",
       "\t3 & 2 & 6 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "GroupedDataFrame with 2 groups based on key: id\n",
       "First Group (3 rows): id = 2\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     2      1\n",
       "   2 │     2      3\n",
       "   3 │     1      5\n",
       "⋮\n",
       "Last Group (3 rows): id = 2\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     2      2\n",
       "   2 │     2      4\n",
       "   3 │     2      6"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[1:3, 1] = [2,2,2]\n",
    "g # well - it is wrong now, g is only a view"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th>Int64</th><th>Int64</th></tr></thead><tbody><p>2 rows × 2 columns</p><tr><th>1</th><td>1</td><td>5</td></tr><tr><th>2</th><td>2</td><td>6</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 5 \\\\\n",
       "\t2 & 2 & 6 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m2×2 SubDataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     1      5\n",
       "   2 │     2      6"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = view(x, 5:6, :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>id</th><th>x</th></tr><tr><th></th><th>Int64</th><th>Int64</th></tr></thead><tbody><p>2 rows × 2 columns</p><tr><th>1</th><td>2</td><td>1</td></tr><tr><th>2</th><td>2</td><td>2</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cc}\n",
       "\t& id & x\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 2 & 1 \\\\\n",
       "\t2 & 2 & 2 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m2×2 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m id    \u001b[0m\u001b[1m x     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼──────────────\n",
       "   1 │     2      1\n",
       "   2 │     2      2"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "delete!(x, 3:6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "BoundsError",
     "evalue": "BoundsError: attempt to access 2-element Array{Int64,1} at index [5:6]",
     "output_type": "error",
     "traceback": [
      "BoundsError: attempt to access 2-element Array{Int64,1} at index [5:6]",
      "",
      "Stacktrace:",
      " [1] throw_boundserror(::Array{Int64,1}, ::Tuple{UnitRange{Int64}}) at .\\abstractarray.jl:541",
      " [2] checkbounds at .\\abstractarray.jl:506 [inlined]",
      " [3] view(::Array{Int64,1}, ::UnitRange{Int64}) at .\\subarray.jl:158",
      " [4] view at D:\\AppData\\.julia\\packages\\DataFrames\\X0xNW\\src\\subdataframe\\subdataframe.jl:133 [inlined]",
      " [5] getindex at D:\\AppData\\.julia\\packages\\DataFrames\\X0xNW\\src\\subdataframe\\subdataframe.jl:170 [inlined]",
      " [6] getindex at D:\\AppData\\.julia\\packages\\DataFrames\\X0xNW\\src\\abstractdataframe\\iteration.jl:189 [inlined]",
      " [7] iterate(::DataFrames.DataFrameColumns{SubDataFrame{DataFrame,DataFrames.Index,UnitRange{Int64}}}, ::Int64) at D:\\AppData\\.julia\\packages\\DataFrames\\X0xNW\\src\\abstractdataframe\\iteration.jl:187 (repeats 2 times)",
      " [8] copyto!(::Array{AbstractArray{T,1} where T,1}, ::DataFrames.DataFrameColumns{SubDataFrame{DataFrame,DataFrames.Index,UnitRange{Int64}}}) at .\\abstractarray.jl:733",
      " [9] _collect at .\\array.jl:630 [inlined]",
      " [10] collect at .\\array.jl:624 [inlined]",
      " [11] broadcastable at .\\broadcast.jl:682 [inlined]",
      " [12] broadcasted at .\\broadcast.jl:1255 [inlined]",
      " [13] _show(::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}, ::SubDataFrame{DataFrame,DataFrames.Index,UnitRange{Int64}}; allrows::Bool, allcols::Bool, rowlabel::Symbol, summary::Bool, eltypes::Bool, rowid::Nothing, truncate::Int64, kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at D:\\AppData\\.julia\\packages\\DataFrames\\X0xNW\\src\\abstractdataframe\\show.jl:227",
      " [14] #show#560 at D:\\AppData\\.julia\\packages\\DataFrames\\X0xNW\\src\\abstractdataframe\\show.jl:480 [inlined]",
      " [15] show(::IOContext{Base.GenericIOBuffer{Array{UInt8,1}}}, ::SubDataFrame{DataFrame,DataFrames.Index,UnitRange{Int64}}) at D:\\AppData\\.julia\\packages\\DataFrames\\X0xNW\\src\\abstractdataframe\\show.jl:480",
      " [16] #show#575 at D:\\AppData\\.julia\\packages\\DataFrames\\X0xNW\\src\\abstractdataframe\\io.jl:50 [inlined]",
      " [17] show at D:\\AppData\\.julia\\packages\\DataFrames\\X0xNW\\src\\abstractdataframe\\io.jl:50 [inlined]",
      " [18] limitstringmime(::MIME{Symbol(\"text/plain\")}, ::SubDataFrame{DataFrame,DataFrames.Index,UnitRange{Int64}}) at D:\\AppData\\.julia\\packages\\IJulia\\ljYVo\\src\\inline.jl:43",
      " [19] display_mimestring at D:\\AppData\\.julia\\packages\\IJulia\\ljYVo\\src\\display.jl:71 [inlined]",
      " [20] display_dict(::SubDataFrame{DataFrame,DataFrames.Index,UnitRange{Int64}}) at D:\\AppData\\.julia\\packages\\IJulia\\ljYVo\\src\\display.jl:102",
      " [21] #invokelatest#1 at .\\essentials.jl:710 [inlined]",
      " [22] invokelatest at .\\essentials.jl:709 [inlined]",
      " [23] execute_request(::ZMQ.Socket, ::IJulia.Msg) at D:\\AppData\\.julia\\packages\\IJulia\\ljYVo\\src\\execute_request.jl:112",
      " [24] #invokelatest#1 at .\\essentials.jl:710 [inlined]",
      " [25] invokelatest at .\\essentials.jl:709 [inlined]",
      " [26] eventloop(::ZMQ.Socket) at D:\\AppData\\.julia\\packages\\IJulia\\ljYVo\\src\\eventloop.jl:8",
      " [27] (::IJulia.var\"#15#18\")() at .\\task.jl:356"
     ]
    }
   ],
   "source": [
    "s # error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Single column selection for `DataFrame` creates aliases with `!` and `getproperty` syntax and copies with `:`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>a</th><th>b</th><th>c</th><th>d</th><th>e</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Int64</th><th>Int64</th><th>Int64</th></tr></thead><tbody><p>3 rows × 5 columns</p><tr><th>1</th><td>1</td><td>1</td><td>1</td><td>1</td><td>1</td></tr><tr><th>2</th><td>2</td><td>2</td><td>2</td><td>2</td><td>2</td></tr><tr><th>3</th><td>3</td><td>3</td><td>3</td><td>3</td><td>3</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccc}\n",
       "\t& a & b & c & d & e\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Int64 & Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 1 & 1 & 1 & 1 & 1 \\\\\n",
       "\t2 & 2 & 2 & 2 & 2 & 2 \\\\\n",
       "\t3 & 3 & 3 & 3 & 3 & 3 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×5 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m a     \u001b[0m\u001b[1m b     \u001b[0m\u001b[1m c     \u001b[0m\u001b[1m d     \u001b[0m\u001b[1m e     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼───────────────────────────────────\n",
       "   1 │     1      1      1      1      1\n",
       "   2 │     2      2      2      2      2\n",
       "   3 │     3      3      3      3      3"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>a</th><th>b</th><th>c</th><th>d</th><th>e</th></tr><tr><th></th><th>Int64</th><th>Int64</th><th>Int64</th><th>Int64</th><th>Int64</th></tr></thead><tbody><p>3 rows × 5 columns</p><tr><th>1</th><td>100</td><td>100</td><td>1</td><td>1</td><td>1</td></tr><tr><th>2</th><td>2</td><td>2</td><td>2</td><td>2</td><td>2</td></tr><tr><th>3</th><td>3</td><td>3</td><td>3</td><td>3</td><td>3</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccc}\n",
       "\t& a & b & c & d & e\\\\\n",
       "\t\\hline\n",
       "\t& Int64 & Int64 & Int64 & Int64 & Int64\\\\\n",
       "\t\\hline\n",
       "\t1 & 100 & 100 & 1 & 1 & 1 \\\\\n",
       "\t2 & 2 & 2 & 2 & 2 & 2 \\\\\n",
       "\t3 & 3 & 3 & 3 & 3 & 3 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m3×5 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m a     \u001b[0m\u001b[1m b     \u001b[0m\u001b[1m c     \u001b[0m\u001b[1m d     \u001b[0m\u001b[1m e     \u001b[0m\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Int64 \u001b[0m\n",
       "─────┼───────────────────────────────────\n",
       "   1 │   100    100      1      1      1\n",
       "   2 │     2      2      2      2      2\n",
       "   3 │     3      3      3      3      3"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = DataFrame(a=1:3)\n",
    "x.b = x[!, 1] # alias\n",
    "x.c = x[:, 1] # copy\n",
    "x.d = x[!, 1][:] # copy\n",
    "x.e = copy(x[!, 1]) # explicit copy\n",
    "display(x)\n",
    "x[1,1] = 100\n",
    "display(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### When iterating rows of a data frame use `eachrow` to avoid compilation cost (wide tables), but `Tables.namedtupleiterator` for fast execution (tall tables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "this table is wide"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th><th>x9</th><th>x10</th><th>x11</th></tr><tr><th></th><th>Char</th><th>Int64</th><th>Char</th><th>Int64</th><th>Float64</th><th>Int64</th><th>Float64</th><th>Float64</th><th>Int64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><p>2 rows × 900 columns (omitted printing of 889 columns)</p><tr><th>1</th><td>a</td><td>1</td><td>a</td><td>1</td><td>1.0</td><td>1</td><td>1.0</td><td>1.0</td><td>1</td><td>1.0</td><td>1.0</td></tr><tr><th>2</th><td>b</td><td>2</td><td>b</td><td>2</td><td>2.0</td><td>2</td><td>2.0</td><td>2.0</td><td>2</td><td>2.0</td><td>2.0</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & x9 & x10 & x11 & \\\\\n",
       "\t\\hline\n",
       "\t& Char & Int64 & Char & Int64 & Float64 & Int64 & Float64 & Float64 & Int64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & a & 1 & a & 1 & 1.0 & 1 & 1.0 & 1.0 & 1 & 1.0 & 1.0 & $\\dots$ \\\\\n",
       "\t2 & b & 2 & b & 2 & 2.0 & 2 & 2.0 & 2.0 & 2 & 2.0 & 2.0 & $\\dots$ \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m2×900 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1   \u001b[0m\u001b[1m x2    \u001b[0m\u001b[1m x3   \u001b[0m\u001b[1m x4    \u001b[0m\u001b[1m x5      \u001b[0m\u001b[1m x6    \u001b[0m\u001b[1m x7      \u001b[0m\u001b[1m x8      \u001b[0m\u001b[1m x9    \u001b[0m\u001b[1m x10 \u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64 \u001b[0m\u001b[90m Float64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Floa\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ a         1  a         1      1.0      1      1.0      1.0  1      1.0  ⋯\n",
       "   2 │ b         2  b         2      2.0      2      2.0      2.0  2      2.0\n",
       "\u001b[31m                                                             891 columns omitted\u001b[0m"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = DataFrame([rand([1:2, 'a':'b', false:true, 1.0:2.0]) for i in 1:900], :auto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0.062215 seconds (112.85 k allocations: 6.044 MiB)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2-element Array{DataFrameRow{DataFrame,DataFrames.Index},1}:\n",
       " \u001b[1mDataFrameRow\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1   \u001b[0m\u001b[1m x2    \u001b[0m\u001b[1m x3   \u001b[0m\u001b[1m x4    \u001b[0m\u001b[1m x5      \u001b[0m\u001b[1m x6    \u001b[0m\u001b[1m x7      \u001b[0m\u001b[1m x8      \u001b[0m\u001b[1m x9    \u001b[0m\u001b[1m x10 \u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64 \u001b[0m\u001b[90m Float64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Floa\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   1 │ a         1  a         1      1.0      1      1.0      1.0  1      1.0  ⋯\n",
       "\u001b[31m                                                             891 columns omitted\u001b[0m\n",
       " \u001b[1mDataFrameRow\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m x1   \u001b[0m\u001b[1m x2    \u001b[0m\u001b[1m x3   \u001b[0m\u001b[1m x4    \u001b[0m\u001b[1m x5      \u001b[0m\u001b[1m x6    \u001b[0m\u001b[1m x7      \u001b[0m\u001b[1m x8      \u001b[0m\u001b[1m x9    \u001b[0m\u001b[1m x10 \u001b[0m ⋯\n",
       "\u001b[1m     \u001b[0m│\u001b[90m Char \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Char \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Float64 \u001b[0m\u001b[90m Float64 \u001b[0m\u001b[90m Int64 \u001b[0m\u001b[90m Floa\u001b[0m ⋯\n",
       "─────┼──────────────────────────────────────────────────────────────────────────\n",
       "   2 │ b         2  b         2      2.0      2      2.0      2.0  2      2.0  ⋯\n",
       "\u001b[31m                                                             891 columns omitted\u001b[0m"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@time collect(eachrow(df1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  3.543209 seconds (5.56 M allocations: 313.787 MiB, 2.72% gc time)\n"
     ]
    }
   ],
   "source": [
    "@time collect(Tables.namedtupleiterator(df1));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "as you can see the time to compile `Tables.namedtupleiterator` is very large in this case, and it would get much worse if the table was wider (that is why we include this tip in pitfalls notebook)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the table below is tall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x1</th><th>x2</th><th>x3</th><th>x4</th><th>x5</th><th>x6</th><th>x7</th><th>x8</th></tr><tr><th></th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th><th>Float64</th></tr></thead><tbody><p>1,000,000 rows × 10 columns (omitted printing of 2 columns)</p><tr><th>1</th><td>0.426059</td><td>0.742032</td><td>0.707272</td><td>0.652059</td><td>0.304092</td><td>0.719528</td><td>0.688369</td><td>0.632429</td></tr><tr><th>2</th><td>0.45075</td><td>0.218993</td><td>0.908634</td><td>0.279011</td><td>0.842885</td><td>0.107019</td><td>0.315626</td><td>0.241803</td></tr><tr><th>3</th><td>0.233813</td><td>0.215836</td><td>0.257729</td><td>0.740415</td><td>0.0919936</td><td>0.71226</td><td>0.914756</td><td>0.726123</td></tr><tr><th>4</th><td>0.965376</td><td>0.898253</td><td>0.95447</td><td>0.672381</td><td>0.83925</td><td>0.724675</td><td>0.150684</td><td>0.396315</td></tr><tr><th>5</th><td>0.184352</td><td>0.357508</td><td>0.528711</td><td>0.966319</td><td>0.917087</td><td>0.90029</td><td>0.230387</td><td>0.956348</td></tr><tr><th>6</th><td>0.927468</td><td>0.542864</td><td>0.978972</td><td>0.632794</td><td>0.19679</td><td>0.213441</td><td>0.752753</td><td>0.186146</td></tr><tr><th>7</th><td>0.479898</td><td>0.176363</td><td>0.831515</td><td>0.223991</td><td>0.3749</td><td>0.83241</td><td>0.497393</td><td>0.356694</td></tr><tr><th>8</th><td>0.560469</td><td>0.413727</td><td>0.95283</td><td>0.00153209</td><td>0.660829</td><td>0.240667</td><td>0.612188</td><td>0.637321</td></tr><tr><th>9</th><td>0.15406</td><td>0.917065</td><td>0.162022</td><td>0.332862</td><td>0.823867</td><td>0.988992</td><td>0.818119</td><td>0.205916</td></tr><tr><th>10</th><td>0.209826</td><td>0.479812</td><td>0.415616</td><td>0.764865</td><td>0.501729</td><td>0.0671917</td><td>0.0618787</td><td>0.0783432</td></tr><tr><th>11</th><td>0.186186</td><td>0.915535</td><td>0.824212</td><td>0.800056</td><td>0.709203</td><td>0.763551</td><td>0.881826</td><td>0.94048</td></tr><tr><th>12</th><td>0.663094</td><td>0.201503</td><td>0.429475</td><td>0.975315</td><td>0.756924</td><td>0.520129</td><td>0.64755</td><td>0.743749</td></tr><tr><th>13</th><td>0.793781</td><td>0.953559</td><td>0.526485</td><td>0.0198605</td><td>0.980669</td><td>0.135033</td><td>0.371642</td><td>0.146413</td></tr><tr><th>14</th><td>0.351356</td><td>0.768198</td><td>0.848575</td><td>0.651162</td><td>0.115418</td><td>0.827824</td><td>0.825066</td><td>0.634127</td></tr><tr><th>15</th><td>0.0285738</td><td>0.465637</td><td>0.397815</td><td>0.961854</td><td>0.374959</td><td>0.806772</td><td>0.0425859</td><td>0.892772</td></tr><tr><th>16</th><td>0.205148</td><td>0.0411017</td><td>0.897482</td><td>0.67378</td><td>0.432806</td><td>0.311889</td><td>0.460766</td><td>0.576969</td></tr><tr><th>17</th><td>0.385248</td><td>0.188962</td><td>0.288653</td><td>0.57601</td><td>0.910782</td><td>0.904109</td><td>0.550595</td><td>0.379574</td></tr><tr><th>18</th><td>0.789076</td><td>0.462469</td><td>0.533758</td><td>0.543552</td><td>0.683761</td><td>0.864939</td><td>0.930492</td><td>0.42189</td></tr><tr><th>19</th><td>0.471257</td><td>0.0802801</td><td>0.989388</td><td>0.433241</td><td>0.599111</td><td>0.834478</td><td>0.385211</td><td>0.161319</td></tr><tr><th>20</th><td>0.81182</td><td>0.206769</td><td>0.428563</td><td>0.763317</td><td>0.0450953</td><td>0.014754</td><td>0.621277</td><td>0.423219</td></tr><tr><th>21</th><td>0.769773</td><td>0.135854</td><td>0.247949</td><td>0.845327</td><td>0.804473</td><td>0.204339</td><td>0.862325</td><td>0.943187</td></tr><tr><th>22</th><td>0.464081</td><td>0.100201</td><td>0.309668</td><td>0.857704</td><td>0.682495</td><td>0.77981</td><td>0.00347277</td><td>0.11548</td></tr><tr><th>23</th><td>0.336477</td><td>0.323941</td><td>0.568314</td><td>0.529947</td><td>0.742448</td><td>0.588876</td><td>0.871196</td><td>0.716296</td></tr><tr><th>24</th><td>0.557822</td><td>0.428115</td><td>0.145344</td><td>0.0121641</td><td>0.561972</td><td>0.492552</td><td>0.67356</td><td>0.204708</td></tr><tr><th>25</th><td>0.311678</td><td>0.48169</td><td>0.783462</td><td>0.223027</td><td>0.495388</td><td>0.400105</td><td>0.847686</td><td>0.0227485</td></tr><tr><th>26</th><td>0.345399</td><td>0.535155</td><td>0.0447053</td><td>0.514991</td><td>0.579781</td><td>0.691561</td><td>0.530411</td><td>0.0433384</td></tr><tr><th>27</th><td>0.296555</td><td>0.289895</td><td>0.19494</td><td>0.713277</td><td>0.69316</td><td>0.976618</td><td>0.499858</td><td>0.705412</td></tr><tr><th>28</th><td>0.462715</td><td>0.782851</td><td>0.705408</td><td>0.200642</td><td>0.722387</td><td>0.359634</td><td>0.507935</td><td>0.678151</td></tr><tr><th>29</th><td>0.621123</td><td>0.672431</td><td>0.868673</td><td>0.444665</td><td>0.26364</td><td>0.475198</td><td>0.510768</td><td>0.572359</td></tr><tr><th>30</th><td>0.900651</td><td>0.190833</td><td>0.365364</td><td>0.326826</td><td>0.900007</td><td>0.142536</td><td>0.339973</td><td>0.613518</td></tr><tr><th>&vellip;</th><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td><td>&vellip;</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ccccccccc}\n",
       "\t& x1 & x2 & x3 & x4 & x5 & x6 & x7 & x8 & \\\\\n",
       "\t\\hline\n",
       "\t& Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & Float64 & \\\\\n",
       "\t\\hline\n",
       "\t1 & 0.426059 & 0.742032 & 0.707272 & 0.652059 & 0.304092 & 0.719528 & 0.688369 & 0.632429 & $\\dots$ \\\\\n",
       "\t2 & 0.45075 & 0.218993 & 0.908634 & 0.279011 & 0.842885 & 0.107019 & 0.315626 & 0.241803 & $\\dots$ \\\\\n",
       "\t3 & 0.233813 & 0.215836 & 0.257729 & 0.740415 & 0.0919936 & 0.71226 & 0.914756 & 0.726123 & $\\dots$ \\\\\n",
       "\t4 & 0.965376 & 0.898253 & 0.95447 & 0.672381 & 0.83925 & 0.724675 & 0.150684 & 0.396315 & $\\dots$ \\\\\n",
       "\t5 & 0.184352 & 0.357508 & 0.528711 & 0.966319 & 0.917087 & 0.90029 & 0.230387 & 0.956348 & $\\dots$ \\\\\n",
       "\t6 & 0.927468 & 0.542864 & 0.978972 & 0.632794 & 0.19679 & 0.213441 & 0.752753 & 0.186146 & $\\dots$ \\\\\n",
       "\t7 & 0.479898 & 0.176363 & 0.831515 & 0.223991 & 0.3749 & 0.83241 & 0.497393 & 0.356694 & $\\dots$ \\\\\n",
       "\t8 & 0.560469 & 0.413727 & 0.95283 & 0.00153209 & 0.660829 & 0.240667 & 0.612188 & 0.637321 & $\\dots$ \\\\\n",
       "\t9 & 0.15406 & 0.917065 & 0.162022 & 0.332862 & 0.823867 & 0.988992 & 0.818119 & 0.205916 & $\\dots$ \\\\\n",
       "\t10 & 0.209826 & 0.479812 & 0.415616 & 0.764865 & 0.501729 & 0.0671917 & 0.0618787 & 0.0783432 & $\\dots$ \\\\\n",
       "\t11 & 0.186186 & 0.915535 & 0.824212 & 0.800056 & 0.709203 & 0.763551 & 0.881826 & 0.94048 & $\\dots$ \\\\\n",
       "\t12 & 0.663094 & 0.201503 & 0.429475 & 0.975315 & 0.756924 & 0.520129 & 0.64755 & 0.743749 & $\\dots$ \\\\\n",
       "\t13 & 0.793781 & 0.953559 & 0.526485 & 0.0198605 & 0.980669 & 0.135033 & 0.371642 & 0.146413 & $\\dots$ \\\\\n",
       "\t14 & 0.351356 & 0.768198 & 0.848575 & 0.651162 & 0.115418 & 0.827824 & 0.825066 & 0.634127 & $\\dots$ \\\\\n",
       "\t15 & 0.0285738 & 0.465637 & 0.397815 & 0.961854 & 0.374959 & 0.806772 & 0.0425859 & 0.892772 & $\\dots$ \\\\\n",
       "\t16 & 0.205148 & 0.0411017 & 0.897482 & 0.67378 & 0.432806 & 0.311889 & 0.460766 & 0.576969 & $\\dots$ \\\\\n",
       "\t17 & 0.385248 & 0.188962 & 0.288653 & 0.57601 & 0.910782 & 0.904109 & 0.550595 & 0.379574 & $\\dots$ \\\\\n",
       "\t18 & 0.789076 & 0.462469 & 0.533758 & 0.543552 & 0.683761 & 0.864939 & 0.930492 & 0.42189 & $\\dots$ \\\\\n",
       "\t19 & 0.471257 & 0.0802801 & 0.989388 & 0.433241 & 0.599111 & 0.834478 & 0.385211 & 0.161319 & $\\dots$ \\\\\n",
       "\t20 & 0.81182 & 0.206769 & 0.428563 & 0.763317 & 0.0450953 & 0.014754 & 0.621277 & 0.423219 & $\\dots$ \\\\\n",
       "\t21 & 0.769773 & 0.135854 & 0.247949 & 0.845327 & 0.804473 & 0.204339 & 0.862325 & 0.943187 & $\\dots$ \\\\\n",
       "\t22 & 0.464081 & 0.100201 & 0.309668 & 0.857704 & 0.682495 & 0.77981 & 0.00347277 & 0.11548 & $\\dots$ \\\\\n",
       "\t23 & 0.336477 & 0.323941 & 0.568314 & 0.529947 & 0.742448 & 0.588876 & 0.871196 & 0.716296 & $\\dots$ \\\\\n",
       "\t24 & 0.557822 & 0.428115 & 0.145344 & 0.0121641 & 0.561972 & 0.492552 & 0.67356 & 0.204708 & $\\dots$ \\\\\n",
       "\t25 & 0.311678 & 0.48169 & 0.783462 & 0.223027 & 0.495388 & 0.400105 & 0.847686 & 0.0227485 & $\\dots$ \\\\\n",
       "\t26 & 0.345399 & 0.535155 & 0.0447053 & 0.514991 & 0.579781 & 0.691561 & 0.530411 & 0.0433384 & $\\dots$ \\\\\n",
       "\t27 & 0.296555 & 0.289895 & 0.19494 & 0.713277 & 0.69316 & 0.976618 & 0.499858 & 0.705412 & $\\dots$ \\\\\n",
       "\t28 & 0.462715 & 0.782851 & 0.705408 & 0.200642 & 0.722387 & 0.359634 & 0.507935 & 0.678151 & $\\dots$ \\\\\n",
       "\t29 & 0.621123 & 0.672431 & 0.868673 & 0.444665 & 0.26364 & 0.475198 & 0.510768 & 0.572359 & $\\dots$ \\\\\n",
       "\t30 & 0.900651 & 0.190833 & 0.365364 & 0.326826 & 0.900007 & 0.142536 & 0.339973 & 0.613518 & $\\dots$ \\\\\n",
       "\t$\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ & $\\dots$ &  \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m1000000×10 DataFrame\u001b[0m\n",
       "\u001b[1m     Row \u001b[0m│\u001b[1m x1       \u001b[0m\u001b[1m x2        \u001b[0m\u001b[1m x3        \u001b[0m\u001b[1m x4         \u001b[0m\u001b[1m x5        \u001b[0m\u001b[1m x6        \u001b[0m\u001b[1m x\u001b[0m ⋯\n",
       "\u001b[1m         \u001b[0m│\u001b[90m Float64  \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64    \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m Float64   \u001b[0m\u001b[90m F\u001b[0m ⋯\n",
       "─────────┼──────────────────────────────────────────────────────────────────────\n",
       "       1 │ 0.426059  0.742032   0.707272   0.652059    0.304092   0.719528   0 ⋯\n",
       "       2 │ 0.45075   0.218993   0.908634   0.279011    0.842885   0.107019   0\n",
       "       3 │ 0.233813  0.215836   0.257729   0.740415    0.0919936  0.71226    0\n",
       "       4 │ 0.965376  0.898253   0.95447    0.672381    0.83925    0.724675   0\n",
       "       5 │ 0.184352  0.357508   0.528711   0.966319    0.917087   0.90029    0 ⋯\n",
       "       6 │ 0.927468  0.542864   0.978972   0.632794    0.19679    0.213441   0\n",
       "       7 │ 0.479898  0.176363   0.831515   0.223991    0.3749     0.83241    0\n",
       "       8 │ 0.560469  0.413727   0.95283    0.00153209  0.660829   0.240667   0\n",
       "       9 │ 0.15406   0.917065   0.162022   0.332862    0.823867   0.988992   0 ⋯\n",
       "      10 │ 0.209826  0.479812   0.415616   0.764865    0.501729   0.0671917  0\n",
       "      11 │ 0.186186  0.915535   0.824212   0.800056    0.709203   0.763551   0\n",
       "    ⋮    │    ⋮          ⋮          ⋮          ⋮           ⋮          ⋮        ⋱\n",
       "  999991 │ 0.586062  0.686106   0.294953   0.655371    0.50385    0.723357   0\n",
       "  999992 │ 0.29458   0.518195   0.719739   0.475906    0.141339   0.590977   0 ⋯\n",
       "  999993 │ 0.479785  0.280123   0.52907    0.906797    0.360529   0.993366   0\n",
       "  999994 │ 0.649908  0.641955   0.395097   0.742147    0.220976   0.0915395  0\n",
       "  999995 │ 0.682031  0.993298   0.209015   0.354809    0.309159   0.592941   0\n",
       "  999996 │ 0.2038    0.784679   0.0263347  0.608568    0.175867   0.547908   0 ⋯\n",
       "  999997 │ 0.768219  0.658602   0.666657   0.11009     0.587625   0.849413   0\n",
       "  999998 │ 0.449748  0.0508578  0.240017   0.299558    0.820843   0.906573   0\n",
       "  999999 │ 0.007136  0.171702   0.269932   0.906575    0.197943   0.787623   0\n",
       " 1000000 │ 0.920932  0.883452   0.779207   0.464268    0.475194   0.84437    0 ⋯\n",
       "\u001b[31m                                               4 columns and 999979 rows omitted\u001b[0m"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = DataFrame(rand(10^6, 10), :auto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  2.127842 seconds (60.20 M allocations: 1.061 GiB, 10.49% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1000000-element Array{Float64,1}:\n",
       " 5.425498600888671\n",
       " 3.7768052553526426\n",
       " 4.659967330231552\n",
       " 7.103370561604825\n",
       " 5.814570363240507\n",
       " 6.1859065433961336\n",
       " 4.471149052140817\n",
       " 5.476184303052102\n",
       " 5.606651743657868\n",
       " 3.47177416802785\n",
       " 6.213498373088708\n",
       " 5.435893647895248\n",
       " 5.0660833071317\n",
       " ⋮\n",
       " 4.725244826624812\n",
       " 6.798193294947637\n",
       " 6.194266127548412\n",
       " 4.206175463282667\n",
       " 5.705487427159872\n",
       " 5.1125101149843895\n",
       " 5.725319376702462\n",
       " 3.463395534254586\n",
       " 4.904474994182661\n",
       " 3.5746522076541454\n",
       " 3.5747126855179996\n",
       " 6.369457105293115"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@time map(sum, eachrow(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  1.968601 seconds (59.99 M allocations: 1.050 GiB, 3.47% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1000000-element Array{Float64,1}:\n",
       " 5.425498600888671\n",
       " 3.7768052553526426\n",
       " 4.659967330231552\n",
       " 7.103370561604825\n",
       " 5.814570363240507\n",
       " 6.1859065433961336\n",
       " 4.471149052140817\n",
       " 5.476184303052102\n",
       " 5.606651743657868\n",
       " 3.47177416802785\n",
       " 6.213498373088708\n",
       " 5.435893647895248\n",
       " 5.0660833071317\n",
       " ⋮\n",
       " 4.725244826624812\n",
       " 6.798193294947637\n",
       " 6.194266127548412\n",
       " 4.206175463282667\n",
       " 5.705487427159872\n",
       " 5.1125101149843895\n",
       " 5.725319376702462\n",
       " 3.463395534254586\n",
       " 4.904474994182661\n",
       " 3.5746522076541454\n",
       " 3.5747126855179996\n",
       " 6.369457105293115"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@time map(sum, eachrow(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0.221442 seconds (516.88 k allocations: 34.874 MiB, 3.61% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1000000-element Array{Float64,1}:\n",
       " 5.425498600888671\n",
       " 3.7768052553526426\n",
       " 4.659967330231552\n",
       " 7.103370561604825\n",
       " 5.814570363240507\n",
       " 6.1859065433961336\n",
       " 4.471149052140817\n",
       " 5.476184303052102\n",
       " 5.606651743657868\n",
       " 3.47177416802785\n",
       " 6.213498373088708\n",
       " 5.435893647895248\n",
       " 5.0660833071317\n",
       " ⋮\n",
       " 4.725244826624812\n",
       " 6.798193294947637\n",
       " 6.194266127548412\n",
       " 4.206175463282667\n",
       " 5.705487427159872\n",
       " 5.1125101149843895\n",
       " 5.725319376702462\n",
       " 3.463395534254586\n",
       " 4.904474994182661\n",
       " 3.5746522076541454\n",
       " 3.5747126855179996\n",
       " 6.369457105293115"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@time map(sum, Tables.namedtupleiterator(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0.020637 seconds (19 allocations: 7.631 MiB)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1000000-element Array{Float64,1}:\n",
       " 5.425498600888671\n",
       " 3.7768052553526426\n",
       " 4.659967330231552\n",
       " 7.103370561604825\n",
       " 5.814570363240507\n",
       " 6.1859065433961336\n",
       " 4.471149052140817\n",
       " 5.476184303052102\n",
       " 5.606651743657868\n",
       " 3.47177416802785\n",
       " 6.213498373088708\n",
       " 5.435893647895248\n",
       " 5.0660833071317\n",
       " ⋮\n",
       " 4.725244826624812\n",
       " 6.798193294947637\n",
       " 6.194266127548412\n",
       " 4.206175463282667\n",
       " 5.705487427159872\n",
       " 5.1125101149843895\n",
       " 5.725319376702462\n",
       " 3.463395534254586\n",
       " 4.904474994182661\n",
       " 3.5746522076541454\n",
       " 3.5747126855179996\n",
       " 6.369457105293115"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@time map(sum, Tables.namedtupleiterator(df2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "as you can see - this time it is much faster to iterate a type stable container"
   ]
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "kernelspec": {
   "display_name": "Julia 1.5.3",
   "language": "julia",
   "name": "julia-1.5"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
